"""
NorskeSkog is the feeder for sensor data provided by Norske Skog.
This feeder iterates through each valid file specified by the
path provided by the delegate configuration, parses those files and 
dispatches all valid samples. 


@author Ben Howell <ben.howell@csiro.au>
"""

import time
import sys
import os
import xml

import string
import threading
from threading import Thread, Event, InterruptedException

from java.lang import Boolean

from com.xhaus.jyson import JysonCodec as json

from org.joda.time import DateTimeZone
from org.joda.time.format import DateTimeFormat 


from au.csiro.ict.tasman.core import Message
from au.csiro.ict.tasman.util import QuickCache
from au.csiro.ict.tasman.datatype import Sample
from sleeper import Sleeper

import xlrd


def run():
  """
  Called by au.csiro.ict.tasman.source.ScriptSource.
  This is the entry point for this script. This call returns the running script
  instance which will, on subsequent calls be held by the "instance" parameter.
  An almost, sort of visitor pattern if you like.
  @return the running script instance.
  """
  return NorskeSkog()
  

def isRunning():
  """
  Called by au.csiro.ict.tasman.source.ScriptSource.
  If false, SourceManager will clean up calling thread and attempt a restart on
  the Java side.
  @return true if running, else false.
  """
  if instance is not None:
    if instance.is_running():
      return Boolean("True")
  return Boolean("False")


def shutDown():
  """
  Called by au.csiro.ict.tasman.source.ScriptSource.
  This allows script to perform its own cleanup routine before exiting.
  @return true after shutdown.
  """
  if instance is not None:
    if instance.shutdown():
      return Boolean("True")
  return Boolean("False")
    


class NorskeSkog:
  """
  Main data source routine.
  """
  def __init__(self):
    """
    Constructor.
    """
    import re
    self.sleeper = Sleeper()
    self.thread_cancelled = False
    self.configuration = json.loads(source.getConfiguration().getConfiguration().toString())
    self.file_name_pattern = re.compile(self.configuration['filenamepattern'])
    self.timestamp_pattern = re.compile(self.configuration['timestamppattern'])
    self.depthmap = self.configuration['depthmap']
    self.header_patterns = self.configuration['headerpatterns']
    self.sheet_formats = self.configuration['sheetformats']
    self.interval = self.configuration['interval']
    self.data_dir = source.getDataDirectory() + self.configuration['datapath']
    self.cache = QuickCache(self.data_dir + "cache.json")
    self.datetime_format = DateTimeFormat.forPattern(self.configuration['timestamp']);
    self.datetime_format = self.datetime_format.withZone(DateTimeZone.UTC)
    self.thread = Thread(target=self.run)
    self.thread.start()
    source.getLogger().info("NorskeSkogSource initialised")
    
    # Norske Skog seem to have numerous standards for constructing
    # spreadsheets. Parameters, columns, headings and uom differ across the
    # years, so let's cater for everything we have seen so far and hope that in
    # future, spreadsheets are constructed using one of the already seen 
    # patterns. Formats below are for heading rows only.
    """
    Formats:
    a.
    Date       Time,        ,           , comment     , parameters     ,             ,             ,        ,   ,
    dd/MM/yyyy H:mm,        , date      ,             , battery        , DO          , Temperature , EC     , pH,
	           ,        ,           ,             , volt           , (ppm)       , (oC)        , (µS/cm),   ,
	           
    b.
    Date/Time      ,        , date      , comment     , battery (volt ), DO          , Temperature , EC     , pH,
    dd/MM/yyyy     ,        ,           ,             ,                , (ppm)       , (C)         , (µS/cm),   ,
    
    c.
    Date       Time, comment, parameters,             ,                ,             ,             ,
	           ,        , battery   , DO__________, Temperature_   , EC__________, pH__________,
	           ,        , volt      , ppm_____    , deg_C___       , us/cm___    , pH______    ,
    """


  def run(self):
    """
    Main loop.
    """
    while not self.thread_cancelled:
      try:
	for file_name in os.listdir(self.data_dir):
	  match = self.file_name_pattern.match(file_name)
	  if match and not self.cache.contains(file_name):
	    self.cache.add(file_name)
	    workbook = os.path.join(self.data_dir, file_name)
	    source.getLogger().debug("parsing workbook: " + workbook)
	    wb = xlrd.open_workbook(workbook)
	    
	    # Norske Skog seem to have no standard for naming or compiling 
	    # sheets. Sometimes it is called "Data", other times it is the 
	    # first unnamed sheet; "Sheet1", on other occasions the first 
	    # unnamed sheet is "Sheet2". Workbooks may also contain any number
	    # of graphs and empty sheets. Let's just try to parse the first 
	    # sheet found as this method works on everything seen so far.
	    
	    worksheet = wb.sheet_by_index(0)
	    source.getLogger().info("validating worksheet: " + worksheet.name)
	    fmt = self.get_format(worksheet)
	    if fmt is not None:
	      source.getLogger().info("sheet format: " + fmt + " detected")
	      self.parse_worksheet(fmt, worksheet, match.group(2))
	    else:
	      source.getLogger().warn("unable to determine header format, skipping worksheet")
	self.cache.save()
	source.getLogger().debug("sleeping...")
	self.sleeper.sleep(self.interval)
	source.getLogger().debug("waking...")
      except InterruptedException:
	self.thread_cancelled = True
	

  def is_running(self):
    """
    Returns true if main loop is running.
    @return true if main loop is running.
    """
    return self.thread.isAlive()
    
  
  def shutdown(self):
    """
    Performs cleanup routine on shutdown.
    @return true after shutdown.
    """
    source.getLogger().debug("shutting down script...")
    self.thread_cancelled = True
    source.getLogger().debug("forcibly waking thread...")
    self.sleeper.waken()
    #block while waiting for thread to terminate
    while self.thread.isAlive():
      time.sleep(1)
    source.getLogger().debug("script terminated!")
    self.cache.save()
    return True
    
    
  def get_format(self, worksheet):
    """
    Validates worksheet and determines the format used.
    """
    import re
    fmt = None
    for f in self.sheet_formats:
      fmt = f
      for field in self.sheet_formats[f]:
	row = self.sheet_formats[f][field][0]
	col = self.sheet_formats[f][field][1]
	if not re.match(self.header_patterns[field], wscell(worksheet,row,col)):
	  fmt = None
	  break
      if not fmt is None:
	return fmt
    return fmt
      

  def parse_worksheet(self, fmt, worksheet, location):
    import re
    
    datetime_col    = self.sheet_formats[fmt]["datetime"][1]
    comment_col     = self.sheet_formats[fmt]["comment"][1]
    battery_col     = self.sheet_formats[fmt]["battery"][1]
    do_col          = self.sheet_formats[fmt]["do"][1]
    temperature_col = self.sheet_formats[fmt]["temperature"][1]
    ec_col          = self.sheet_formats[fmt]["ec"][1]
    ph_col          = self.sheet_formats[fmt]["ph"][1]
    
    messages = list()
    altitude = float(self.depthmap[location.lower()]) * -1
    network_feature = "norske_skog.boyer."
    
    for nrow in range(self.wsheader_nrows(fmt), worksheet.nrows):
      cells = worksheet.row_values(nrow)
      
      if re.match(self.timestamp_pattern, str(cells[datetime_col])):
	datetime = self.datetime_format.parseDateTime(cells[datetime_col])
	comment = cells[comment_col]
	
	if cells[battery_col] is not None and cells[battery_col] != "":
	  fq_sensor_id = network_feature + "battery_" + str(self.depthmap[location.lower()]) + "m"
	  messages.append(create_message(datetime, fq_sensor_id, "V", cells[battery_col], altitude, comment))
	  
	if cells[do_col] is not None and cells[do_col] != "":
	  fq_sensor_id = network_feature + "do_" + str(self.depthmap[location.lower()]) + "m"
	  messages.append(create_message(datetime, fq_sensor_id, "ppm", cells[do_col], altitude, comment))
	  
	if cells[temperature_col] is not None and cells[temperature_col] != "":
	  fq_sensor_id = network_feature + "temperature_" + str(self.depthmap[location.lower()]) + "m"
	  messages.append(create_message(datetime, fq_sensor_id, "degrees C", cells[temperature_col], altitude, comment))
	  
	if cells[ec_col] is not None and cells[ec_col] != "":
	  fq_sensor_id = network_feature + "ec_" + str(self.depthmap[location.lower()]) + "m"
	  messages.append(create_message(datetime, fq_sensor_id, "uS/cm", cells[ec_col], altitude, comment))
	  
	if cells[ph_col] is not None and cells[ph_col] != "":
	  fq_sensor_id = network_feature + "ph_" + str(self.depthmap[location.lower()]) + "m"
	  messages.append(create_message(datetime, fq_sensor_id, "pH", cells[ph_col], altitude, comment))
	
      # Be nice to the dispatcher, batch up the messages. 
      if len(messages) > 19:
	source.getLogger().debug(" dispatching " + str(len(messages)) + " messages")
	source.dispatch(messages)
	del messages[:]
      
    # Dispatch any remaining messages. 
    if messages:
      source.getLogger().debug("** cleaning up, dispatching final " + str(len(messages)) + " messages")
      source.dispatch(messages)
      del messages[:]
    source.getLogger().debug("worksheet parsed")
      
    
  def wsheader_nrows(self, fmt):
    n_rows = -1
    for field in self.sheet_formats[fmt]:
      if self.sheet_formats[fmt][field][0] > n_rows:
	n_rows = self.sheet_formats[fmt][field][0]
    return n_rows+1
    

def create_message(datetime, fq_sensor_id, uom, value, altitude, comment):
  s = Sample()
  s.time = datetime
  if type(value) != type(float):
    value = float(value)
  s.value = value
  s.sensorId = fq_sensor_id
  s.tags.put("uom", uom)
  if altitude is not None:
    s.tags.put("altitude", str(altitude))
  if comment is not None:
    s.tags.put("comment", comment)
  return Message(s.sensorId, s)

  
def equal(a, b):
  try:
    return a.lower() == b.lower()
  except AttributeError:
    return a == b
 
 
def wscell(worksheet, row, col):
  if worksheet.cell_type(row,col) in (xlrd.XL_CELL_EMPTY, xlrd.XL_CELL_BLANK):
    return ""
  elif worksheet.cell_type(row,col) in (xlrd.XL_CELL_NUMBER, xlrd.XL_CELL_DATE, xlrd.XL_CELL_BOOLEAN, xlrd.XL_CELL_TEXT):
    return worksheet.cell(row,col).value
  elif worksheet.cell_type(row,col) == xlrd.XL_CELL_ERROR:
    source.getLogger().error("spreadsheet cell error: " + xlrd.error_text_from_code(xlrd.XL_CELL_ERROR)) 
    return None
